In [113]:
#!pip install -q pycountry
#!pip install gensim
#!pip install textblob
#!pip install wordcloud
#!pip install plotly
In [114]:
from scipy import stats
import warnings
warnings.filterwarnings("ignore")
import math
import numpy as np
import scipy as sp
import pandas as pd
import pycountry
from sklearn import metrics
from sklearn.utils import shuffle
from gensim.models import Word2Vec
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
import nltk
from textblob import TextBlob
from wordcloud import WordCloud
from nltk.corpus import wordnet
from nltk.corpus import stopwords
from nltk import WordNetLemmatizer
from nltk.stem import WordNetLemmatizer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import random
import networkx as nx
from pandas import Timestamp
import requests
from IPython.display import HTML
In [115]:
import seaborn as sns
from tqdm import tqdm
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
from plotly.subplots import make_subplots
In [116]:
tqdm.pandas()
np.random.seed(0)
%env PYTHONHASHSEED=0
import warnings
warnings.filterwarnings("ignore")
In [117]:
biorxiv_df = pd.read_csv('D:/GSUCoursework/BigDataExp/Datasets/cleaned csv/biorxiv_clean.csv')
pmc_df = pd.read_csv('D:/GSUCoursework/BigDataExp/Datasets/cleaned csv/clean_pmc.csv')
comm_use_df = pd.read_csv('D:/GSUCoursework/BigDataExp/Datasets/cleaned csv/clean_comm_use.csv')
noncomm_use_df = pd.read_csv('D:/GSUCoursework/BigDataExp/Datasets/cleaned csv/clean_noncomm_use.csv')
In [118]:
papers_df = pd.concat([pmc_df,
biorxiv_df,
comm_use_df,
noncomm_use_df], axis=0).reset_index(drop=True)
In [119]:
papers_df
Out[119]:
In [120]:
papers_df['authors']
Out[120]:
In [121]:
full_table = pd.read_csv('D:/GSUCoursework/BigDataExp/Datasets/Covid_19_clean/Covid_19_latest/covid_19_clean_complete.csv')
In [122]:
full_table.head(10)
Out[122]:
Data Cleaning¶
In [123]:
# Converting Date column to datetime datatype
full_table.dtypes
Out[123]:
In [124]:
full_table['Date'] = pd.to_datetime(full_table['Date'])
In [125]:
full_table.dtypes
Out[125]:
In [126]:
# Checking for null values
full_table.isna().sum()
Out[126]:
In [127]:
# filling missing values
full_table['Province/State'] = full_table['Province/State'].fillna('')
In [128]:
full_table.isna().sum()
Out[128]:
In [129]:
# replacing Mainland china with just China
full_table['Country/Region'] = full_table['Country/Region'].replace('Mainland China', 'China')
In [130]:
# Creating a new column 'Active' which will represent all the present active cases
full_table['Active'] = full_table['Confirmed'] - full_table['Deaths'] - full_table['Recovered']
In [131]:
full_table.head(10)
Out[131]:
Data Preprocessing¶
In [132]:
# Cases in the ships
ship = full_table[full_table['Province/State'].str.contains('Grand Princess')|full_table['Country/Region'].str.contains('Cruise Ship')]
In [133]:
# Let us seperate out only China data into variable China and all other countries_Region into variable row
# china and the row
china = full_table[full_table['Country/Region']=='China']
row = full_table[full_table['Country/Region']!='China']
In [134]:
# latest cases
full_latest = full_table[full_table['Date'] == max(full_table['Date'])].reset_index()
china_latest = full_latest[full_latest['Country/Region']=='China']
row_latest = full_latest[full_latest['Country/Region']!='China']
In [135]:
# latest condensed
full_latest_grouped = full_latest.groupby('Country/Region')['Confirmed', 'Deaths', 'Recovered', 'Active'].sum().reset_index()
china_latest_grouped = china_latest.groupby('Province/State')['Confirmed', 'Deaths', 'Recovered', 'Active'].sum().reset_index()
row_latest_grouped = row_latest.groupby('Country/Region')['Confirmed', 'Deaths', 'Recovered', 'Active'].sum().reset_index()
Exploratory Data Analysis¶
Current Situation¶
In [136]:
# Creating a consolidated table , which gives the country wise total defined cases
temp = full_table.groupby(['Country/Region', 'Province/State'])['Confirmed', 'Deaths', 'Recovered', 'Active'].max()
temp = full_table.groupby('Date')['Confirmed', 'Deaths', 'Recovered', 'Active'].sum().reset_index()
temp = temp[temp['Date']==max(temp['Date'])].reset_index(drop=True)
temp.style.background_gradient(cmap='Pastel1')
Out[136]:
There are more recovered cases than deaths at this point of time¶
Countries which have been affected by the Coronavirus(2019-nCoV)till now¶
In [137]:
countries = full_table['Country/Region'].unique().tolist()
print(countries)
print("\nTotal countries affected by virus: ",len(countries))
For visualizations, we are using Plotly Python Open Source Graphing Library. Plotly is another great Python visualization tool that’s capable of handling geographical, scientific, statistical, and financial data. Plotly has several advantages over matplotlib. One of the main advantages is that only a few lines of codes are necessary to create aesthetically pleasing, interactive plots. The interactivity also offers a number of advantages over static matplotlib plots:¶¶
1) Saves time when initially exploring your dataset
2) Makes it easy to modify and export your plot
3) Offers a more ornate visualization, which is well-suited for conveying the important insights hidden within your dataset
Pie Plot¶
In [138]:
'''A Function To Plot Pie Plot using Plotly'''
def pie_plot(cnt_srs, colors, title):
labels=cnt_srs.index
values=cnt_srs.values
trace = go.Pie(labels=labels,
values=values,
title=title,
hoverinfo='percent+value',
textinfo='percent',
textposition='inside',
hole=0.7,
showlegend=True,
marker=dict(colors=colors,
line=dict(color='#000000',
width=2),
)
)
return trace
In [139]:
'''Plotly visualization'''
import plotly.offline as py
from plotly.offline import iplot, init_notebook_mode
import plotly.graph_objs as go
py.init_notebook_mode(connected = True) # Required to use plotly offline in jupyter notebook
In [140]:
py.iplot([pie_plot(full_table['Country/Region'].value_counts(), ['cyan', 'gold'], 'Country')])
In [141]:
# World wide
temp = full_table[full_table['Date'] == max(full_table['Date'])]
m = folium.Map(location=[0, 0], tiles='cartodbpositron',
min_zoom=1, max_zoom=4, zoom_start=1)
for i in range(0, len(temp)):
folium.Circle(
location=[temp.iloc[i]['Lat'], temp.iloc[i]['Long']],
color='crimson', fill='crimson',
tooltip = '<li><bold>Country : '+str(temp.iloc[i]['Country/Region'])+
'<li><bold>Province : '+str(temp.iloc[i]['Province/State'])+
'<li><bold>Confirmed : '+str(temp.iloc[i]['Confirmed'])+
'<li><bold>Deaths : '+str(temp.iloc[i]['Deaths']),
radius=int(temp.iloc[i]['Confirmed'])**1.1).add_to(m)
m
Out[141]:
In [ ]:
In [142]:
full_grouped = full_table.groupby(['Date', 'Country/Region'])['Confirmed', 'Deaths', 'Recovered', 'Active'].sum().reset_index()
In [143]:
# Over the time
fig = px.choropleth(full_grouped, locations="Country/Region", locationmode='country names', color=np.log(full_grouped["Confirmed"]),
hover_name="Country/Region", animation_frame=full_grouped["Date"].dt.strftime('%Y-%m-%d'),
title='Cases over time', color_continuous_scale=px.colors.sequential.Magenta)
fig.update(layout_coloraxis_showscale=False)
fig.show()
Top 20 countries having most confirmed cases¶
In [144]:
top = full_table[full_table['Date'] == full_table['Date'].max()]
top_casualities = top.groupby(by = 'Country/Region')['Confirmed'].sum().sort_values(ascending = False).head(20).reset_index()
top_casualities
Out[144]:
In [145]:
plt.figure(figsize= (15,10))
plt.xticks(fontsize = 15)
plt.yticks(fontsize = 15)
plt.xlabel("Total cases",fontsize = 30)
plt.ylabel('Country',fontsize = 30)
plt.title("Top 20 countries having most confirmed cases" , fontsize = 30)
ax = sns.barplot(x = top_casualities['Confirmed'], y = top_casualities['Country/Region'])
for i, (value, name) in enumerate(zip(top_casualities['Confirmed'],top_casualities['Country/Region'])):
ax.text(value, i-.05, f'{value:,.0f}', size=10, ha='left', va='center')
ax.set(xlabel='Total cases', ylabel='Country')
Out[145]:
Observations :
1) China was leading this from many days, but now they are controlling the pandemic spread.
2) The number of confirmed cases are on a high in the US, Italy, Spain, and France.
3) But the number of cases in the third world countries is less.
Top 20 countries having most active cases¶
In [146]:
top_actives = top.groupby(by = 'Country/Region')['Active'].sum().sort_values(ascending = False).head(20).reset_index()
top_actives
Out[146]:
In [147]:
plt.figure(figsize= (15,10))
plt.xticks(fontsize = 15)
plt.yticks(fontsize = 15)
plt.xlabel("Total cases",fontsize = 30)
plt.ylabel('Country',fontsize = 30)
plt.title("Top 20 countries having most active cases" , fontsize = 30)
ax = sns.barplot(x = top_actives['Active'], y = top_actives['Country/Region'])
for i, (value, name) in enumerate(zip(top_actives['Active'], top_actives['Country/Region'])):
ax.text(value, i-.05, f'{value:,.0f}', size=10, ha='left', va='center')
ax.set(xlabel='Total cases', ylabel='Country')
Out[147]:
Observations :
1) As the covid-19 testing is increasing, The active number of cases is also increasing day by day.
2) The number of active cases is on a high in the US, Italy, Spain, and France.
3) In comparison with the total population, Italy, Spain, Germany and France are the worst-hit nation
Top 20 countries having most deaths¶
In [148]:
top_deaths = top.groupby(by = 'Country/Region')['Deaths'].sum().sort_values(ascending = False).head(20).reset_index()
top_deaths
Out[148]:
In [149]:
plt.figure(figsize= (15,10))
plt.xticks(fontsize = 15)
plt.yticks(fontsize = 15)
plt.xlabel("Total cases",fontsize = 30)
plt.ylabel('Country',fontsize = 30)
plt.title("Top 20 countries having most deaths" , fontsize = 30)
ax = sns.barplot(x = top_deaths['Deaths'], y = top_deaths['Country/Region'])
for i, (value, name) in enumerate(zip(top_deaths['Deaths'],top_deaths['Country/Region'])):
ax.text(value, i-.05, f'{value:,.0f}', size=10, ha='left', va='center')
ax.set(xlabel='Total cases', ylabel='Country')
Out[149]:
Observations :
1) Even though Italy has the 2nd best healthcare system according to the WHO, they haven't been able to tackle the pandemic problem effectively.
2) China even having so many confirmed cases was able to decrease the number of deaths
3) The number of deaths is also on a rise, especially in Italy, Spain, and Iran
Top 20 countries having most recovered cases¶
In [150]:
top_recovered = top.groupby(by = 'Country/Region')['Recovered'].sum().sort_values(ascending = False).head(20).reset_index()
top_recovered
Out[150]:
In [151]:
plt.figure(figsize= (15,10))
plt.xticks(fontsize = 15)
plt.yticks(fontsize = 15)
plt.xlabel("Total cases",fontsize = 30)
plt.ylabel('Country',fontsize = 30)
plt.title("Top 20 countries having most recovered cases" , fontsize = 30)
ax = sns.barplot(x = top_recovered['Recovered'], y = top_recovered['Country/Region'])
for i, (value, name) in enumerate(zip(top_recovered['Recovered'],top_recovered['Country/Region'])):
ax.text(value, i-.05, f'{value:,.0f}', size=10, ha='left', va='center')
ax.set(xlabel='Total cases', ylabel='Country')
Out[151]:
1) By far China was leading in the number of recoveries even though having a huge number of confirmed cases, but recently Germany has surpassed China in terms of most recovered cases. No wonder Germany has best healthcare facilities
2) Italy, Iran, and Spain are also doing a good job.
3) We have to pump up these numbers for a promising future!
Author names¶
Every research paper in the corpus is written by one or more authors, and the names of these authors could provide some insights regarding which parts of the world generate the bulk of coronavirus research.¶
We used an API called Namsor to predict the country of origin of the authors. Note: The predictions made by this API are not 100% accurate and sometimes it is difficult to predict someone's country of origin solely based on the name. So, this will atleast give us a brief idea about which parts of the world are generating bulk of research on Coronavirus¶
In [152]:
from IPython.display import Image
Image(filename = 'D:/GSUCoursework/BigDataExp/Datasets/Pictures/Namesor.PNG', width = 500, height = 500)
Out[152]:
In [153]:
#!pip install namsor-client
In [154]:
from namsorclient import NamsorClient
from namsorclient.country_codes import CountryCodes
from namsorclient.request_objects import *
# Create an instance of NamsorClient and pass in your API key as an argument.
client = NamsorClient("3ab91fa7404d5bfa2dcd4a183f037cc7")
# Access the origin (GET) endpoint with function that returns a response of type Country of origin.
response = client.origin("Lelouch","Lamperouge")
# Access the different parts of the response for this particular endpoint.
print(response.ID)
print(response.first_name)
print(response.last_name)
print(response.country_origin)
#print(response.gender_scale)
print(response.score)
#print(response.probability_calibrated)
#response = client.origin(repl[0], repl[1])
#repls.append(response.country_origin)
# Refer to the Responses section to view all different variables of each different Response.
# Access the genderBatch (POST) endpoint
origin_batch = OriginBatch()
# Use classify function with required API Key argument and receive responses in the form of a list.
response_list = origin_batch.classify("3ab91fa7404d5bfa2dcd4a183f037cc7")
#print(response_list[2].country_origin)
In [ ]:
#client = NamsorClient("3ab91fa7404d5bfa2dcd4a183f037cc7")
def multiple_replace(dict, text):
regex = re.compile("(%s)" % "|".join(map(re.escape, dict.keys())))
return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text)
def get_countries(raw_authors):
alphabet = ["A", "B", "C", "D", "E", "F",\
"G", "H", "I", "J", "K", "L",\
"M", "N", "O", "P", "Q", "R",\
"S", "T", "U", "V", "W", "X",\
"Y", "Z"]
repl_dict = dict(zip([a+" " for a in alphabet], [""]*26))
repls = []
for name in names.split(", "):
repl = multiple_replace(repl_dict, name.strip().replace(") ", "").replace("( ", ""))
if len(repl.split()) == 1:
repl = name[0] + " " + repl
repl = repl.replace(";", "").replace(":", "").replace(".", "").replace(",", "")
repl = repl.split(" ")
for idx in range(len(repl)):
if len(repl[idx]) <= 1 and repl[idx] not in alphabet:
repl[idx] = "A"
response = client.origin(repl[0], repl[1])
repls.append(response.country_origin)
return repls
In [ ]:
origin_batch.export_to_excel("countries.xlsx")
In [ ]:
In [ ]:
Unsupervised NLP and Word2Vec¶
Unsupervised NLP involves the analysis of unlabeled language data. Certain techniques can be used to derive insights from a large corpus of text. One such method is called Word2Vec. Word2Vec is a neural network architecture trained on thousands of sentences of text. After training, the neural network finds the optimal vector representation of each word in the corpus. These vectors are meant to reflect the meaning of the word. Words with similar meanings have similar vectors.¶
In [156]:
from IPython.display import Image
Image(filename = 'D:/GSUCoursework/BigDataExp/Datasets/Pictures/Word2Vec_Basic.PNG', width = 700, height = 800)
Out[156]:
As I stated earlier, each word is associated with a vector. Amazingly, these vectors can also encode relationships and analogies between words. The diagram below iillustrates some examples of linear vector relationships representing the relationships between words.¶
In [157]:
from IPython.display import Image
Image(filename = 'D:/GSUCoursework/BigDataExp/Datasets/Pictures/Wrod2Vec_Relationships.PNG', width = 800, height = 700)
Out[157]:
In the above image, we can see that word vectors can reflect relationships such as "King is to Queen as Man is to Woman" or "Italy is to Rome" as "Germany is to Berlin". These vectors can be also be used to find unknown relationships between words. These unknown relationships may help us find latent knowledge in research papers and find drugs that can possibly cure COVID_19!¶
Using Word2Vec to find cures¶
We can take advantage of these intricate relationships between word vectors to find cures for COVID-19. The steps are as follows:
Step 1 - Find common related to the study of COVID-19, such as "infection", "CoV", "viral", etc.
Step 2 - Find the words with lowest Euclidean distance to these words (most similar words).
Step 3 - Finally, find the words most similar to these words (second order similarity). These words will hopefully contain potential COVID-19 cures.
Note that the similarity between two Word2Vec vectors is calculated using the formula below (where u and v are the word vectors).
In [158]:
from IPython.display import Image
Image(filename = 'D:/GSUCoursework/BigDataExp/Datasets/Pictures/Word2Vec_Formula.PNG', width = 500, height = 500)
Out[158]:
The entire process can be summarized with the flowchart below. (the same steps as given above)¶
In [159]:
from IPython.display import Image
Image(filename = 'D:/GSUCoursework/BigDataExp/Datasets/Pictures/NLP_three_Steps.PNG', width = 500, height = 500)
Out[159]:
The approach detailed above is actually inspired by a research paper called "Unsupervised word embeddings capture latent knowledge from materials science literature", where the authors find new materials with desirable properties (such as thermoelectricity) solely based on a large corpus materials science literature. These materials were never used for these purposes before, but they outperform old materials by a large margin. I hope to emulate the same method to look for COVID-19 cures. The diagram below illustrates what the authors did in their research.
In [160]:
from IPython.display import Image
Image(filename = 'D:/GSUCoursework/BigDataExp/Datasets/Pictures/Thermoelectricity.PNG', width = 500, height = 500)
Out[160]:
In the diagram above, we can see that the authors found two levels of words similar to "thermoelectric" in a heirarchical manner. The second order similar words contained compounds like Li2CuSb, Cu7Te5, and CsAgGa2Se4, which turned out to be very good thermoelectric materials in real life.
First, we need to find the most common words in the corpus to continue our analysis. To find the most common words, we can create a word cloud¶
Word cloud of abstracts¶
In [161]:
def nonan(x):
if type(x) == str:
return x.replace("\n", "")
else:
return ""
text = ' '.join([nonan(abstract) for abstract in papers_df["abstract"]])
wordcloud = WordCloud(max_font_size=None, background_color='white', collocations=False,
width=1200, height=1000).generate(text)
fig = px.imshow(wordcloud)
fig.update_layout(title_text='Common words in abstracts')